﻿using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Data;
using System.Drawing;
using System.Net;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
using System.Windows.Forms;

namespace Assignment10
{
    public partial class Form1 : Form
    {
        public static int i = 0;
        public Form1()
        {
            InitializeComponent();
            visitedUrls = new HashSet<string>();
            errorUrls = new HashSet<string>();
            client = new WebClient();
            client.Encoding = Encoding.UTF8; // 设置编码为UTF-8
            
        }

        private void Form1_Load(object sender, EventArgs e)
        {

        }
        private HashSet<string> UrlsToCrawl = new HashSet<string>();
        private HashSet<string> visitedUrls = new HashSet<string>() ;  // 已访问的URL集合
        private HashSet<string> errorUrls = new HashSet<string>();    // 错误的URL集合
        private string baseUrl;               // 起始URL
        private WebClient client;             // WebClient对象用于获取网页内容

        private void Crawl(string url)
        {
            while (true)
            {
                if (UrlsToCrawl.Count >= 15) return;
                if (visitedUrls.Contains(url)) return;
                lock (UrlsToCrawl)
                {
                    if (!Regex.Match(url, "^https?://").Success)
                    {
                        url = "http://" + url;
                    }
                    UrlsToCrawl.Add(url);
                    listBox1.Items.Add(url);
                }
                string html = null;
                try
                {
                    html = client.DownloadString(url);
                }
                catch (Exception ex)
                {
                    errorUrls.Add(url);
                    listBox2.Items.Add(url + " (" + ex.Message + ")");
                    Console.WriteLine(ex.Message);
                    return;
                }
                visitedUrls.Add(url);

                var links = GetLinks(html);
                foreach (var link in links)
                {
                    if (!IsAbsoluteUrl(link))
                    {
                        var absoluteLink = ConvertToAbsoluteUrl(link, url);
                        if (absoluteLink != null && absoluteLink.StartsWith(baseUrl))
                        {
                            if (IsValidUrl(absoluteLink))
                            {
                                Crawl(absoluteLink);
                            }
                            else
                            {
                                errorUrls.Add(absoluteLink);
                                listBox1.Items.Add("[错误] " + absoluteLink);
                            }
                        }
                    }
                    else if (IsValidUrl(link))
                    {
                        Crawl(link);
                    }
                    else
                    {
                        errorUrls.Add(link);
                        listBox1.Items.Add("[错误] " + link);
                    }

                }
            }
        }
        
    private List<string> GetLinks(string html)
        {
            var links = new List<string>();
            string strRef = @"(href|HREF)\s*=\s*[""'](?<url>[^""'#>]+)[""']";
            var regex = new Regex(strRef);
            var matches = regex.Matches(html);
            
            foreach (Match match in matches)
            {
                links.Add(match.Groups["url"].Value);
                
            }
            return links;
        }

        private bool IsAbsoluteUrl(string url)
        {
            return url.StartsWith("http://") || url.StartsWith("https://");
        }

        private string ConvertToAbsoluteUrl(string url, string baseUrl)
        {
            Uri baseUri = new Uri(baseUrl);
            Uri absoluteUri = null;
            if (Uri.TryCreate(baseUri, url, out absoluteUri))
            {
                return absoluteUri.ToString();
            }
            else
            {
                return null;
            }
        }
        private static bool IsValidUrl(string url)
        {
            string pattern = @".*\.(htm|html|aspx|php|jsp)$";
            return Regex.IsMatch(url, pattern, RegexOptions.IgnoreCase);
        }
        private void button1_Click(object sender, EventArgs e)
        {
            baseUrl = textBox1.Text.Trim();
            visitedUrls.Clear();
            errorUrls.Clear();
            listBox1.Items.Clear();
            listBox2.Items.Clear();
            for (int i = 0; i < 5; i++)
            {
                Thread th = new Thread(delegate () { Crawl(baseUrl); });
                th.Start();
            }
            
            
        }

        private void textBox1_TextChanged(object sender, EventArgs e)
        {

        }
    }
    }
